{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Experiments_MNIST.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mlelarge/dataflowr/blob/master/Notebooks/04_deeper/04_dropout_mnist_colab.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "ETPdoc-2yEl0",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Dropout"
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "sx9e_pXlCuti",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "%load_ext autoreload\n",
        "%autoreload 2"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "BdUwwL-3yEl6",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import random\n",
        "\n",
        "%matplotlib inline\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import torch\n",
        "from torch.optim import lr_scheduler\n",
        "import torch.optim as optim\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torchvision\n",
        "from torchvision.datasets import MNIST\n",
        "from torchvision import transforms\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "text",
        "id": "UMMut8UVCutt"
      },
      "cell_type": "markdown",
      "source": [
        "# 1. Setup and initializations\n",
        "We'll go through analysing the role of Dropout on MNIST dataset. \n"
      ]
    },
    {
      "metadata": {
        "id": "ljzioFzByEl-",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "class ExperimentParams():\n",
        "    def __init__(self):\n",
        "        self.data_dir = '/home/docker_user/'\n",
        "        self.num_classes = 10\n",
        "        self.device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "        self.batch_size = 256\n",
        "        self.num_epochs = 20\n",
        "        self.num_workers = 4\n",
        "        self.lr = 1e-2\n",
        "        \n",
        "        self.drop_prob1 = 0.2\n",
        "        self.drop_prob2 = 0.5\n",
        "\n",
        "args = ExperimentParams()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "text",
        "id": "BcmGBqXeCutw"
      },
      "cell_type": "markdown",
      "source": [
        "## 1.1 Prepare dataset\n"
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "AQSHPH9P0BNx",
        "colab": {},
        "outputId": "cbb81b98-d4bb-4d36-9989-9572225d0e99"
      },
      "cell_type": "code",
      "source": [
        "mean, std = 0.1307, 0.3081\n",
        "\n",
        "train_dataset = MNIST(f'{args.data_dir}/data/MNIST', train=True, download=True,\n",
        "                             transform=transforms.Compose([\n",
        "                                 transforms.ToTensor(),\n",
        "                                 transforms.Normalize((mean,), (std,))\n",
        "                             ]))\n",
        "test_dataset = MNIST(f'{args.data_dir}/data/MNIST', train=False, download=True,\n",
        "                            transform=transforms.Compose([\n",
        "                                transforms.ToTensor(),\n",
        "                                transforms.Normalize((mean,), (std,))\n",
        "                            ]))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "error",
          "ename": "OSError",
          "evalue": "[Errno 45] Operation not supported: '/home/docker_user'",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-4-04471a49ecea>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m                              transform=transforms.Compose([\n\u001b[1;32m      5\u001b[0m                                  \u001b[0mtransforms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mToTensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m                                  \u001b[0mtransforms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mNormalize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mstd\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m                              ]))\n\u001b[1;32m      8\u001b[0m test_dataset = MNIST(f'{args.data_dir}/data/MNIST', train=False, download=True,\n",
            "\u001b[0;32m/Users/abursuc/anaconda3/lib/python3.6/site-packages/torchvision-0.2.1-py3.6.egg/torchvision/datasets/mnist.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, root, train, transform, target_transform, download)\u001b[0m\n\u001b[1;32m     44\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     45\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mdownload\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 46\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdownload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_exists\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/Users/abursuc/anaconda3/lib/python3.6/site-packages/torchvision-0.2.1-py3.6.egg/torchvision/datasets/mnist.py\u001b[0m in \u001b[0;36mdownload\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    102\u001b[0m         \u001b[0;31m# download files\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 104\u001b[0;31m             \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmakedirs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraw_folder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    105\u001b[0m             \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmakedirs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprocessed_folder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    106\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/Users/abursuc/anaconda3/lib/python3.6/os.py\u001b[0m in \u001b[0;36mmakedirs\u001b[0;34m(name, mode, exist_ok)\u001b[0m\n\u001b[1;32m    208\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhead\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mtail\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexists\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    209\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 210\u001b[0;31m             \u001b[0mmakedirs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexist_ok\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    211\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mFileExistsError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    212\u001b[0m             \u001b[0;31m# Defeats race condition when another thread created the path\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/Users/abursuc/anaconda3/lib/python3.6/os.py\u001b[0m in \u001b[0;36mmakedirs\u001b[0;34m(name, mode, exist_ok)\u001b[0m\n\u001b[1;32m    208\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhead\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mtail\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexists\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    209\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 210\u001b[0;31m             \u001b[0mmakedirs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexist_ok\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    211\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mFileExistsError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    212\u001b[0m             \u001b[0;31m# Defeats race condition when another thread created the path\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/Users/abursuc/anaconda3/lib/python3.6/os.py\u001b[0m in \u001b[0;36mmakedirs\u001b[0;34m(name, mode, exist_ok)\u001b[0m\n\u001b[1;32m    208\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhead\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mtail\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexists\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    209\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 210\u001b[0;31m             \u001b[0mmakedirs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexist_ok\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    211\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mFileExistsError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    212\u001b[0m             \u001b[0;31m# Defeats race condition when another thread created the path\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/Users/abursuc/anaconda3/lib/python3.6/os.py\u001b[0m in \u001b[0;36mmakedirs\u001b[0;34m(name, mode, exist_ok)\u001b[0m\n\u001b[1;32m    218\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    219\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 220\u001b[0;31m         \u001b[0mmkdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    221\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    222\u001b[0m         \u001b[0;31m# Cannot rely on checking for EEXIST, since the operating system\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mOSError\u001b[0m: [Errno 45] Operation not supported: '/home/docker_user'"
          ]
        }
      ]
    },
    {
      "metadata": {
        "colab_type": "text",
        "id": "TcZTFRnjCut3"
      },
      "cell_type": "markdown",
      "source": [
        "## 1.2 Common setup"
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "Dz2xh66UCut5",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "\n",
        "mnist_classes = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']\n",
        "\n",
        "\n",
        "def get_raw_images(dataloader,mean=0.1307, std=0.3081):\n",
        "\n",
        "    raw_images = np.zeros((len(dataloader.dataset), 1, 28, 28))\n",
        "    k = 0\n",
        "    for input, target in dataloader:\n",
        "        raw_images[k:k+len(input)] = (input*std + mean).data.cpu().numpy()\n",
        "        k += len(input)\n",
        "\n",
        "    return raw_images\n",
        "\n",
        "\n",
        "def show(img, title=None):\n",
        "    # img is a torch.Tensor     \n",
        "    npimg = img.numpy()\n",
        "    plt.imshow(np.transpose(npimg, (1,2,0)), interpolation='nearest')\n",
        "    plt.axis('off')\n",
        "    if title is not None:\n",
        "        plt.title(title)\n",
        "    plt.pause(0.001)  # pause a bit so that plots are updated"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "text",
        "id": "75moY8AyCut_"
      },
      "cell_type": "markdown",
      "source": [
        "# 2. Playing with DropOut\n"
      ]
    },
    {
      "metadata": {
        "id": "_6n5CYbbyEmM",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## 2.1 Architecture"
      ]
    },
    {
      "metadata": {
        "id": "TJ9uTbUUyEmN",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#### Exercise\n",
        "\n",
        "Complete the missing blocks in the definition of the following `DropoutNet` architecture: (`FullyConnected 256 -> ReLU -> Dropout (0.2) -> Fully Connected 256 -> ReLU -> -> Dropout (0.5) -> Fully Connected 10 `)"
      ]
    },
    {
      "metadata": {
        "id": "Ud4O5WVnyEmN",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "class DropoutNet(nn.Module):\n",
        "    def __init__(self, num_classes=10,drop_prob1=0.2, drop_prob2=0.5):\n",
        "        super(DropoutNet, self).__init__()\n",
        "        self.classifier = nn.Sequential( \n",
        "                    #  TODO\n",
        "                    )\n",
        "\n",
        "    def forward(self, x):\n",
        "                # TODO\n",
        "        return self.classifier(x)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "iMwq9oISyEmQ",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## 2.2 Training"
      ]
    },
    {
      "metadata": {
        "id": "I262fl6AyEmQ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Set up data loaders\n",
        "\n",
        "kwargs = {'num_workers': args.num_workers, 'pin_memory': True} \n",
        "train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, **kwargs)\n",
        "test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=args.batch_size, shuffle=False, **kwargs)\n",
        "\n",
        "model_dropout = DropoutNet(num_classes=args.num_classes, drop_prob1=args.drop_prob1, drop_prob2=args.drop_prob2)\n",
        "optimizer_dropout = optim.Adam(model_dropout.parameters(), lr=args.lr)\n",
        "scheduler_dropout = lr_scheduler.StepLR(optimizer_dropout, 8, gamma=0.1, last_epoch=-1)\n",
        "\n",
        "model_simple = DropoutNet(num_classes=args.num_classes, drop_prob1=0, drop_prob2=0)\n",
        "optimizer_simple = optim.Adam(model_simple.parameters(), lr=args.lr)\n",
        "scheduler_simple = lr_scheduler.StepLR(optimizer_dropout, 8, gamma=0.1, last_epoch=-1)\n",
        "\n",
        "loss_fn = torch.nn.CrossEntropyLoss()\n",
        "\n",
        "model_dropout.to(args.device)\n",
        "model_simple.to(args.device)\n",
        "loss_fn.to(args.device)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "GPpqOL4ryEmS",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def train_classif_epoch(train_loader, model, loss_fn, optimizer, args, log_interval=100):\n",
        "    model.train()\n",
        "    losses = []\n",
        "    total_loss, total_corrects, num_samples = 0, 0, 0\n",
        "    corrects = 0.    \n",
        "    for batch_idx, (data, target) in enumerate(train_loader):\n",
        "        num_samples += data.size(0)\n",
        "        \n",
        "        data, target = data.to(args.device), target.to(args.device)\n",
        "        \n",
        "        optimizer.zero_grad()\n",
        "        outputs = model(data)\n",
        "\n",
        "        loss = loss_fn(outputs, target)\n",
        "        losses.append(loss.data.item())\n",
        "\n",
        "        _,preds = torch.max(outputs.data,1)\n",
        "        corrects += torch.sum(preds == target.data).cpu()\n",
        "\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        \n",
        "        if batch_idx % log_interval == 0:\n",
        "            print('Train: [{}/{} ({:.0f}%)]\\tLoss: {:.6f} \\tAccuracy: {}'.format(\n",
        "                batch_idx * len(data[0]), len(train_loader.dataset),\n",
        "                100. * batch_idx / len(train_loader), np.mean(losses), float(total_corrects)/num_samples))           \n",
        "            \n",
        "            total_loss += np.sum(losses)\n",
        "            total_corrects += corrects\n",
        "            losses, corrects = [], 0\n",
        "            \n",
        "    accuracy = total_corrects.item()/num_samples\n",
        "    return total_loss/(batch_idx + 1), accuracy\n",
        "\n",
        "def test_classif_epoch(test_loader, model, loss_fn, args, log_interval=100):\n",
        "    with torch.no_grad():\n",
        "        model.eval()\n",
        "        losses, corrects = [], 0\n",
        "        num_samples = 0\n",
        "        corrects = 0.\n",
        "        for batch_idx, (data, target) in enumerate(test_loader):\n",
        "\n",
        "            num_samples += data.size(0)\n",
        "            data, target = data.to(args.device), target.to(args.device)\n",
        "\n",
        "            outputs = model(data)\n",
        "\n",
        "            loss = loss_fn(outputs, target)\n",
        "            losses.append(loss.data.item())\n",
        "\n",
        "            _,preds = torch.max(outputs.data,1)\n",
        "            corrects += torch.sum(preds == target.data).cpu()\n",
        "\n",
        "        accuracy = corrects.item()/num_samples\n",
        "        return np.sum(losses)/(batch_idx + 1), accuracy"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "mZYE3GuHyEmV",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#### Training the baseline model for a while"
      ]
    },
    {
      "metadata": {
        "id": "DQdcrSBcyEmV",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "start_epoch = 0\n",
        "\n",
        "for epoch in range(0, start_epoch):\n",
        "    scheduler_simple.step()\n",
        "\n",
        "train_losses_simple, val_losses_simple, val_accuracies_simple = [], [], []\n",
        "for epoch in range(start_epoch, args.num_epochs):\n",
        "    scheduler_simple.step()\n",
        "\n",
        "    train_loss, train_accuracy = train_classif_epoch(train_loader, model_simple, loss_fn, optimizer_simple, args)\n",
        "\n",
        "    message = 'Epoch: {}/{}. Train set: Average loss: {:.4f} Average accuracy: {:.4f}'.format(\n",
        "        epoch + 1, args.num_epochs, train_loss, train_accuracy)\n",
        "    \n",
        "    val_loss, val_accuracy = test_classif_epoch(test_loader, model_simple, loss_fn, args)\n",
        "    \n",
        "    message += '\\nEpoch: {}/{}. Validation set: Average loss: {:.4f}  Average accuracy: {:.4f}'.format(epoch + 1, args.num_epochs,\n",
        "                                                                             val_loss, val_accuracy)\n",
        "    print(message)\n",
        "    train_losses_simple.append(train_loss)\n",
        "    val_losses_simple.append(val_loss)\n",
        "    val_accuracies_simple.append(val_accuracy)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "m2d1WpIzyEmX",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#### Training the Dropout variant"
      ]
    },
    {
      "metadata": {
        "id": "MwjoLIifyEmY",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "start_epoch = 0\n",
        "\n",
        "for epoch in range(0, start_epoch):\n",
        "    scheduler_dropout.step()\n",
        "\n",
        "train_losses, val_losses, val_accuracies = [], [], []\n",
        "for epoch in range(start_epoch, args.num_epochs):\n",
        "    scheduler_dropout.step()\n",
        "\n",
        "    train_loss, train_accuracy = train_classif_epoch(train_loader, model_dropout, loss_fn, optimizer_dropout, args)\n",
        "\n",
        "    message = 'Epoch: {}/{}. Train set: Average loss: {:.4f} Average accuracy: {:.4f}'.format(\n",
        "        epoch + 1, args.num_epochs, train_loss, train_accuracy)\n",
        "    \n",
        "    val_loss, val_accuracy = test_classif_epoch(test_loader, model_dropout, loss_fn, args)\n",
        "    \n",
        "    message += '\\nEpoch: {}/{}. Validation set: Average loss: {:.4f}  Average accuracy: {:.4f}'.format(epoch + 1, args.num_epochs,\n",
        "                                                                             val_loss, val_accuracy)\n",
        "    print(message)\n",
        "    train_losses.append(train_loss)\n",
        "    val_losses.append(val_loss)\n",
        "    val_accuracies.append(val_accuracy)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "jB08xAdqyEma",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "\n",
        "#### Plot results"
      ]
    },
    {
      "metadata": {
        "id": "W8DFj5zcyEmb",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "plt.cla()\n",
        "epochs = np.arange(args.num_epochs)\n",
        "plt.plot(epochs, train_losses_simple, 'orange', lw=3, label='no dropout')\n",
        "plt.plot(epochs, train_losses, 'green', lw=3, label='with dropout')\n",
        "plt.legend(loc='upper left'); \n",
        "plt.xlabel('epoch')\n",
        "plt.ylabel('train loss')\n",
        "plt.title('Train loss')\n",
        "plt.grid(True)\n",
        "plt.pause(0.1)\n",
        "plt.show()\n",
        "\n",
        "\n",
        "plt.cla()\n",
        "epochs = np.arange(args.num_epochs)\n",
        "plt.plot(epochs, val_losses_simple, 'orange', lw=3, label='no dropout')\n",
        "plt.plot(epochs, val_losses, 'green', lw=3, label='with dropout')\n",
        "plt.legend(loc='upper left'); \n",
        "plt.xlabel('epoch')\n",
        "plt.ylabel('validation loss')\n",
        "plt.title('Validation loss')\n",
        "plt.grid(True)\n",
        "plt.pause(0.1)\n",
        "plt.show()\n",
        "\n",
        "        \n",
        "plt.cla()\n",
        "epochs = np.arange(args.num_epochs)\n",
        "plt.plot(epochs, val_accuracies_simple, 'orange', lw=3, label='no dropout')\n",
        "plt.plot(epochs, val_accuracies, 'green', lw=3, label='with dropout')\n",
        "plt.legend(loc='upper left'); \n",
        "plt.xlabel('epoch')\n",
        "plt.ylabel('validation accuracy')\n",
        "plt.title('Validation accuracy')\n",
        "plt.grid(True)\n",
        "plt.pause(0.1)\n",
        "plt.show()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "J5Ctf7y1yEmd",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# 3. Exercises"
      ]
    },
    {
      "metadata": {
        "id": "liyeEOxQyEme",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "1. Try out what happens if you change the dropout probabilities for layers 1 and 2. In particular, what happens if you switch the ones for both layers?\n",
        "2. Increase the number of epochs and compare the results obtained when using dropout with those when not using it.\n",
        "3. If changes are made to the model to make it more complex, such as adding hidden layer units, will the effect of using dropout to cope with overfitting be more obvious?\n",
        "4. What happens if we apply dropout to the individual weights of the weight matrix rather than the activations?\n"
      ]
    },
    {
      "metadata": {
        "id": "t7j-G1glyEmf",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}